package com.chance.cc.crawler.development.bootstrap.autohome.distributor;

import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.filter.FilterUtils;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.development.controller.DevCrawlerController;

import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerRequestType.*;

public class AutohomeDistributorStart {

    public static void main(String[] args) {
        String domain = "autohome";
        String site = "carSeriesDistributor";
        String url="https://dealer.autohome.com.cn/shanghai/";
//        String url="https://dealer.autohome.com.cn/shanghai/0/0/0/0/1/1/0/0.html";

        CrawlerRequestRecord crawlerRequestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domain, CrawlerEnum.CrawlerRequestType.turnPage)
                .domain(domain)
                .httpUrl(url)
                .recordKey(url)
                .releaseTime(System.currentTimeMillis())
                .filter(CrawlerEnum.CrawlerRecordFilter.keyOrDateRange)
                .addFilterInfo(FilterUtils.memoryFilterKeyInfo(domain))
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(24, null))
                .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                .build();
        crawlerRequestRecord.tagsCreator().bizTags().addCustomKV("firstUrl",url);
        crawlerRequestRecord.tagsCreator().bizTags().addDomain(domain);
        crawlerRequestRecord.tagsCreator().bizTags().addSite(site);

        CrawlerRequestRecord keywordRecord = CrawlerRequestRecord.builder()
                .startPageRequest("autohome_series_keyword",turnPageItem)
                .httpUrl("http://192.168.1.215:9599/v1/meta/autohome/keys?site=dealer-simple")
                .requestLabelTag(supportSource)
                .requestLabelTag(internalDownload)
                .build();

        DevCrawlerController.builder()
                .triggerInfo(domain, domain, System.currentTimeMillis(), domain)//本地的内存队列  放置脚本
                .crawlerRequestQueue(DevCrawlerController.devRequestQueue(domain))//定义本地内存方式 创建队列
                .crawlerThreadNum(30)
                .consoleResultPipeline()//控制台输出
                .fileResultPipeline("D:\\chance\\log\\car4.log", false)//文件输出   true为追加文件  false为覆盖
                .requestRecord(crawlerRequestRecord)//创建队列 放入requestRecord调用DevCrawlerController启动
                .supportRecord(keywordRecord)//加载关键字
//                .supportRecord(cityRecord)
                .build("com.chance.cc.crawler.development.scripts.autohome")
                .start();
    }
}